import numpy as np
import pandas as pd
import sklearn.pipeline
from sklearn.kernel_approximation import RBFSampler
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import params
import random
import matplotlib.pyplot as plt
# parameters stored in params.py
workload_max = 30
workload_min = 10
number_VM_max = 11
number_VM_min = 2
response_time_intercept = [-338.8173500, -99.5766850, 185.0698700, 257.5648000, 397.3988700, 375.8291500, 371.0434800, 423.1175400, 377.6604100, 368.1021800]
response_time_coefficients = [310.9830100, 196.6712300, 113.6043900, 91.8822210, 66.1391060, 62.3167310, 58.4447950, 52.3450650, 53.0440920, 52.8421350]
utility_fixed_revenue = 30
utility_coefficients = [1.0, 200/2000]
response_time_SLA = 2000
# I include all the relevant codes in this file. The codes are from multiple files. You will need to make necessary adjustments, e.g. separating them back to multiple files or changing name spaces.
# function to calculate response time
def calculate_response_time(number_VM, workload):
i = number_VM - params.number_VM_min
response_time = params.response_time_intercept[i] + params.response_time_coefficients[i] * workload
return response_time
# read and scale the work loads
raw_workload_list = np.genfromtxt('milano_callin_r.txt', skip_header = 1).flatten(1)[0:8502,]
raw_workload_max = raw_workload_list.max()
raw_workload_min = raw_workload_list.min()
workload_scaling = (params.workload_max - params.workload_min)/(raw_workload_max - raw_workload_min)
workload_list = (raw_workload_list - raw_workload_min) * workload_scaling + params.workload_min
workload_length = 8502
# function to calculate rewards
def utility_linear_cost(metrics):
utility = params.utility_fixed_revenue - (params.utility_coefficients[0] * metrics["number_VM"] + params.utility_coefficients[1] * max(metrics["response_time"] - params.response_time_SLA, 0))
return utility
class cloud:
def __init__(self,initial_wl,initial_vm):
#all parameters to be accessed using params.py
#params.workload_max
#init states
self.num_vm = initial_vm
self.wl = initial_wl
self.initial_state = (initial_wl,initial_vm)
self.rew = 0
#init action space
self.actions = [-2,-1,0,1,2] #-2 means scaling down number of vms by 2.
print("Current state : " , self.initial_state)
print("Possible actions: " , self.actions)
print("Current number of VMs: " , self.num_vm)
def reset_env(self):
self.wl,self.num_vm = self.initial_state
#self.initial_state = (self.wl,self.num_vm)
return self.initial_state
def action_space(self,action):
act_adjusted = 'NA'
#scale down number of VMs by 2
if action == 0:
projected_num_vm = self.num_vm - 2
if projected_num_vm < params.number_VM_min:
act_adjusted = self.num_vm - params.number_VM_min
print('Action -2 not possible so Scaled down by',act_adjusted)
self.num_vm = self.num_vm - act_adjusted
if act_adjusted == 0:
act_adjusted = 2
else:
act_adjusted = 1
else:
self.num_vm = projected_num_vm
#scale down number of VMs by 1
if action == 1:
projected_num_vm = self.num_vm - 1
if projected_num_vm < params.number_VM_min:
act_adjusted = self.num_vm - params.number_VM_min
print('Action -2 not possible so Scaled down by',act_adjusted)
self.num_vm = self.num_vm - act_adjusted
if act_adjusted == 0:
act_adjusted = 2
else:
act_adjusted = 1
else:
self.num_vm = projected_num_vm
#scale up/down number of VMs by 0 i.e. do nothing
if action == 2:
pass #do nothing
#scale up number of VMs by 1
if action == 3:
projected_num_vm = self.num_vm + 1
if projected_num_vm > params.number_VM_max:
act_adjusted = params.number_VM_max - self.num_vm
print('Action +1 not possible so Scaled up by',act_adjusted)
self.num_vm = self.num_vm + act_adjusted
#map action into 0,1,2,3
if act_adjusted == 0:
act_adjusted = 2
else:
act_adjusted = 3
else:
self.num_vm = projected_num_vm
#scale up number of VMs by 2
if action == 4:
projected_num_vm = self.num_vm + 2
if projected_num_vm > params.number_VM_max:
act_adjusted = params.number_VM_max - self.num_vm
print('Action +2 not possible so Scaled up by',act_adjusted)
self.num_vm = self.num_vm + act_adjusted
if act_adjusted == 0:
act_adjusted = 2
else:
act_adjusted = 3
else:
self.num_vm = projected_num_vm
if act_adjusted != 'NA':
action = act_adjusted
return self.num_vm,action
def reward(self,curr_wl_state,curr_num_vm):
#take action and decide the reward
#fix the wl to current wl
#next_num_vm,act = self.action_space(action)
reward = params.utility_fixed_revenue - (params.utility_coefficients[0] * curr_num_vm + params.utility_coefficients[1]
* max(calculate_response_time(curr_num_vm,curr_wl_state) - params.response_time_SLA, 0))
return reward
def step(self,action,curr_wl_state,curr_num_vm):
curr_RT = calculate_response_time(curr_num_vm,curr_wl_state)
#rw = self.reward(curr_wl_state,curr_num_vm)
#self.rew = rw
#run and get the reward,state
next_num_vm,act = self.action_space(action)
#also return response time
next_ind = np.where(workload_list == curr_wl_state)[0][0] + 1
if next_ind == len(workload_list):
next_ind = 0
nxt_RT = calculate_response_time(next_num_vm,workload_list[next_ind])
rw = self.reward(workload_list[next_ind],next_num_vm)
self.rew = rw
print("Current State,action,reward,Response time,Next State: " , (curr_num_vm,curr_wl_state),action,self.rew,curr_RT
,(workload_list[next_ind],next_num_vm))
#return state,action,reward,next state
return (curr_wl_state,curr_num_vm),act,self.rew,nxt_RT,(workload_list[next_ind],next_num_vm)
## Build features for input states
## mapping them into high dimensions would produce a better estimate using neural networks
# builds mercer kernel
featurizer = sklearn.pipeline.FeatureUnion([
("rbf1", RBFSampler(gamma=5.0, n_components=100)),
("rbf2", RBFSampler(gamma=2.0, n_components=100)),
("rbf3", RBFSampler(gamma=1.0, n_components=100)),
("rbf4", RBFSampler(gamma=0.5, n_components=100))
])
featurizer.fit(np.random.choice(workload_list,1000).reshape(-1,1))
#data_set = featurizer.transform(workload_list.reshape(-1,1))
class NNet(nn.Module):
"""Initializes a simple sequential Neural network model
To do : complete the doc """
def __init__(self,in_dimension = 401, num_hidden_nodes = 20,action_space = 5):
# N is batch size; D_in is input dimension;
# H is hidden dimension; D_out is output dimension.
super(NNet,self).__init__()
#N = x_train.shape[0] not necessary to provide its dynamic
D_in, H, D_out = in_dimension, num_hidden_nodes, action_space
# Create random Tensors to hold inputs and outputs
#self.x = torch.tensor(x_train).float()
#self.y = torch.tensor(y_train).float()
self.h1 = nn.Linear(D_in,H)
self.out = nn.Linear(H,D_out)
#declare a loss function
#self.loss_fn = torch.nn.MSELoss(size_average=False)
def forward(self,x):
x = self.h1(x)
x = F.tanh(x)
x = self.out(x)
return x
""" Fit the model using backprop
Input X_train and target vals Y """
def fit(self,x_tr,y_tr,learning_rate = 0.004,iterations = 5000):
opt = optim.Adam(params=obj.parameters(), lr= learning_rate)
loss_list = []
for i in range(iterations):
res = self(x_tr) #out values
_,pred = res.max(1) #prediction
loss = F.mse_loss(res,y_tr) #calculate loss between output and target value
loss_list.append(loss)
print('loss' , loss.item())
self.zero_grad()
loss.backward()
opt.step()
return res,pred,loss_list
""" Converts input state to featurized and predicts the q values for a given state. Only to be used inside
epsilon greedy policy """
def predict(self,state):
w,n = state
# w = self.state_to_highdim(w)
w = torch.tensor(np.append(self.state_to_highdim(w),n)).float()
res = self(w)
res = res.reshape(1,res.shape[0])
return res.max(1)[1].item()
def state_to_highdim(self,val):
tn = featurizer.transform(val)
return torch.tensor(tn).float()
""" Print model parameters"""
def print_model_params(self):
for name, param in self.named_parameters():
if param.requires_grad:
print (name, param.data)
learning_rate = 0.001
opt = optim.Adam(params=current_net.parameters(), lr= learning_rate)
loss_list = []
def optimizer():
#target net , current net
batch_dct = mem_obj.sample_batch()
if(batch_dct==None):return
batch_actions = []
batch_reward = []
batch_states_wl = [] #current state workload
batch_states_numvm = [] #current state number of vms
batch_next_states_wl = [] #next state workload
batch_next_states_numvm = [] #next state numvm
for i in range(len(batch_dct)):
batch_states_wl.append(batch_dct[i][0][0])
batch_states_numvm.append(batch_dct[i][0][1])
batch_actions.append(batch_dct[i][1])
batch_reward.append(batch_dct[i][2])
batch_next_states_wl.append(batch_dct[i][3][0])
batch_next_states_numvm.append(batch_dct[i][3][1])
#compute a forward pass with current states
vec = current_net.state_to_highdim(np.array(batch_states_wl).reshape(-1,1))
tn_vec = torch.tensor(np.concatenate((vec,np.array(batch_states_numvm).reshape(len(batch_dct),1)),axis = 1)).float()
batch_predictions = current_net(tn_vec)
action_indices = np.array(batch_actions).reshape(len(batch_actions),1)
q_estimated = batch_predictions.gather(1,torch.tensor(action_indices)) #select q values according to the actions taken
batch_target_predictions = torch.zeros(len(batch_dct), device=device)
vec2 = current_net.state_to_highdim(np.array(batch_next_states_wl).reshape(-1,1))
tn_vec2 = torch.tensor(np.concatenate((vec2,np.array(batch_next_states_numvm).reshape(len(batch_dct),1)),axis = 1)).float()
batch_target_predictions = target_net(tn_vec2).max(1)[0].detach() #tensor of max q values length = batch size
q_targets = (batch_target_predictions * GAMMA) + torch.tensor(batch_reward)
#now optimise the loss between q targets and q estimated
loss = F.mse_loss(q_targets,q_estimated) #calculate loss between output and target value
loss_list.append(loss)
print('loss' , loss.item())
opt.zero_grad()
loss.backward()
opt.step()
class memory:
def __init__(self,batch_size = 100,mem_size = 1000):
self.mem_size = mem_size
self.batch_size = batch_size
self.mem_dct = {}
for i in range(mem_size):
self.mem_dct[i] = []
self.counter = 0
self.mem_flag = 0
def update_memory(self,input_list): #state,action,reward,next state
if(self.counter!=self.mem_size):
self.mem_dct[self.counter] = input_list
self.counter = self.counter + 1
else:
self.counter = 0 # rewrite the oldest entry
self.mem_flag = 1 # to indicate if memory is full
self.mem_dct[self.counter] = input_list
def sample_batch(self):
if(self.mem_flag==1):
rand_ind = np.random.choice(self.mem_size,self.batch_size,replace=False) #sample 100 random numbers from the memory size of 1000
batch_dct = {}
cnt = 0
for key in rand_ind:
batch_dct[cnt] = self.mem_dct[key] #store sampled batch in batch_dct
cnt = cnt + 1
else:return None
return batch_dct
mem_obj = memory(10,20)
# epsilon-greedy policy
def eps_greedy(epsilon,state):
rn = random.uniform(0,1)
# maximize
if(rn > epsilon):
action = current_net.predict(state)
else:
#print('random action')
action = np.random.choice([0,1,2,3,4]) # 0th action = -2 , 1st action = -1 , 2nd action = 0 , 3rd action = 1, 4th action = 2
return action
### DECLARE ALL CLASS OBJECTS ######
#Neural network objects
current_net = NNet()
target_net = NNet()
#memory object
mem_obj = memory()
#environment object
cloud_obj = cloud(workload_list[15],10)
#q learning variables
#hyper parameters
total_episodes = 1000 # Total episodes
#learning_rate = 0.8 # Learning rate
max_steps = 100 # Max steps per episode
GAMMA = 0.5 # Discounting rate
# Exploration parameters
epsilon = 1.0 # Exploration rate
max_epsilon = 1.0 # Exploration probability at start
min_epsilon = 0.01 # Minimum exploration probability
decay_rate = 0.01 # Exponential decay rate for exploration prob
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
episodic_reward = [] #list of rewards at the end of each episode
all_steps = []
curr_state_list = []
reward_list = []
action_list = []
RT_list = []
for i in range(1,total_episodes):
#at beginning of every episode reset the environment
state = cloud_obj.reset_env()
total_rewards = 0
total_steps = 0
count = 0
print("############ Running episode number: " + str(i) +" ##############")
for step in range(max_steps):
action = eps_greedy(epsilon,state) #choose action according to epsilon-greedy
_,curr_num_vm = state
#get the current workload
if count==workload_length:
count = 0
curr_wl = workload_list[count]
count = count + 1
# now take a step in the environment
state,action,rew,RT,next_state = cloud_obj.step(action,curr_wl,curr_num_vm) #1returns next state, reward
#store transition
mem_obj.update_memory([state,action,rew,next_state])
##optimize model
optimizer()
total_rewards = total_rewards + rew
curr_state_list.append(state)
reward_list.append(rew) #reward at every time step
action_list.append(action)
RT_list.append(RT)
state = next_state
#if(done=='True'):
# break
#update target net
if i % 10 == 0:
target_net.load_state_dict(current_net.state_dict())
epsilon = 1*np.exp(-decay_rate*i)
episodic_reward.append(total_rewards)
all_steps.append(step)